# -*- coding: utf-8 -*-
import scrapy
from mafengwo import items
import json
from scrapy import Selector

class MayoujiSpider(scrapy.Spider):
    name = "mayouji"
    allowed_domains = ["www.mafengwo.cn"] 
    start_urls = (
        'http://www.mafengwo.cn/mdd/',
    )
    prefix = "http://www.mafengwo.cn"


    def getYJContent(self, response):
        youji = items.MaYouji()
        youji['url'] = response.url
        youji['ding'] = response.meta['ding']
        youji['cid'] = response.meta['cid']
        end = response.url.rfind('.html')
        start = response.url.rfind('/')
        yid = response.url[start+1:end]
        youji['yid'] = yid
        raw = response.selector.xpath('//div[@id="pnl_contentinfo"]/*')
        #raw = response.selector.xpath('/html/body/div[4]/div[2]').extract()
        out = []
        for item in raw:
            txt = item.xpath('text()|.//*/text()|.//*/*/text()').extract()
            if len(txt) > 0:
                out.append(txt[0])
        youji['content'] = "".join(out)
        yield youji
    
    def parseYjForPage(self, response):
        yjitems = response.selector.xpath('//div[@class="post-list"]/ul/li')
        for item in yjitems:
            number_ding = item.xpath('div[@class="post-ding"]/span/text()').extract()[0] 
            url = item.xpath('h2/a/@href').extract()[0]
            turl = self.prefix + url
            yield scrapy.Request(self.prefix + url, callback=self.getYJContent, meta = {'ding': number_ding, 'cid': response.meta['cid']}, dont_filter=True)

    def parseYouji(self, response):
        pid = 1
        yjitems = response.selector.xpath('//div[@class="post-list"]/ul/li')
        for item in yjitems:
            number_ding = item.xpath('div[@class="post-ding"]/span/text()').extract()[0] 
            url = item.xpath('h2/a/@href').extract()[0]
            turl = self.prefix + url
            yield scrapy.Request(self.prefix + url, callback=self.getYJContent, meta = {'ding': number_ding, 'cid': response.meta['cid']}, dont_filter=True)
        #TODO parse other pages
        pgnum = response.selector.xpath('//div[@class="page-hotel"]/span[@class="count"]/span/text()').extract()
        if len(pgnum) > 0:
            pgcnt = int(pgnum[0])
            for p in range(2, pgcnt):
                url = "http://www.mafengwo.cn/yj/%s/1-0-%d.html" % (response.meta['cid'], p)
                yield scrapy.Request(url, callback=self.parseYjForPage, meta=response.meta)

    def parseJDForPage(self, response):
        jdlist = response.selector.xpath('//div[@class="content"]/div/div/ul/li')
        for jditem in jdlist:
            jd = items.MaJD()
            jd['cname'] = jditem.xpath('.//div[@class="title"]/h3/a/text()').extract()
            jd['cid'] = response.meta['cid']
            jd['url'] =  jditem.xpath('.//div[@class="title"]/h3/a/@href').extract()
            jd['imageurl'] = jditem.xpath('.//span/a/img/@src').extract()
            jd['score'] = jditem.xpath('.//div[@class="grade"]/em/text()').extract()
            jd['rank'] = jditem.xpath('.//div[@class="title"]/div[@class="ranking"]/em/text()').extract()
            yield jd


    def parseJD(self, response):
        pid = 1
        
        jdlist = response.selector.xpath('//div[@class="content"]/div/div/ul/li')
        for jditem in jdlist:
            jd = items.MaJD()
            jd['cid'] = response.meta['cid']
            jd['cname'] = jditem.xpath('.//div[@class="title"]/h3/a/text()').extract()
            jd['url'] =  jditem.xpath('.//div[@class="title"]/h3/a/@href').extract()
            jd['imageurl'] = jditem.xpath('.//span/a/img/@src').extract()
            jd['score'] = jditem.xpath('.//div[@class="grade"]/em/text()').extract()
            jd['rank'] = jditem.xpath('.//div[@class="title"]/div[@class="ranking"]/em/text()').extract()
            yield jd

        
        pgnumber = response.selector.xpath('//div[@class="page-hotel"]/span[@class="count"]/span[1]/text()').extract()
        print "xxxxxxxxxxxxxxxxxxxxxxxxx pgnum xxxxxxxxxxxxxxxxxxxx", pgnumber, response.url
        if len(pgnumber) > 0:
            pgcnt = int(pgnumber[0])
            prefix = "http://www.mafengwo.cn/jd/%s/0-0-0-0-0-%d.html"
            for p in range(2, pgcnt):
                url = prefix % (response.meta['cid'], p)
                yield scrapy.Request(url, callback=self.parseJDForPage, meta=response.meta)

        
    def parseHotelForPage(self, response):
        if len(response.body) == 0:
            return
        resp = json.loads(response.body)
        selector = Selector(text=resp['html'])
        hlist = selector.xpath('//div/dl')
        for h in hlist:
            hotel = items.MaHotel()
            hotel['url'] = h.xpath('.//dt/h3/a/@href').extract()
            hotel['cid'] = response.meta['cid']
            name = h.xpath('.//@data-name').extract()
            if len(name) > 0:
                start = name[0].rfind('(')
                hotel['cname'] = name[0][0:start]
                end  = name[0].rfind(')')
                hotel['ename'] = name[0][start+1:end]

            hotel['lat'] = h.xpath('.//@data-lat').extract()
            hotel['lng'] = h.xpath('.//@data-lng').extract()
            hotel['addr'] = h.xpath('.//div[@class="address"]/p/@title').extract()
            yield hotel

    def parseHotel(self, response):
        pid = 1
        hlist = response.selector.xpath('//*[@id="hotel_list"]/div[1]/dl')
        for h in hlist:
            hotel = items.MaHotel()
            hotel['url'] = h.xpath('.//dt/h3/a/@href').extract()
            hotel['cid'] = response.meta['cid']
            name = h.xpath('.//@data-name').extract()
            if len(name) > 0:
                start = name[0].rfind('(')
                hotel['cname'] = name[0][0:start]
                end  = name[0].rfind(')')
                hotel['ename'] = name[0][start+1:end]

            hotel['lat'] = h.xpath('.//@data-lat').extract()
            hotel['lng'] = h.xpath('.//@data-lng').extract()
            hotel['addr'] = h.xpath('.//div[@class="address"]/p/@title').extract()
            yield hotel
        
        #TODO other pages
        pgnum = response.selector.xpath('//*[@id="list_paginator"]/span[1]/span[1]/text()').extract()
        if len(pgnum) > 0:
            pgcnt = int(pgnum[0])
            prefix = "http://www.mafengwo.cn/hotel/ajax.php?sAction=getPoiList2&iMddId=%s&sKeyWord=&iPage=%d&sTags=&iPriceMin=0&iPriceMax=&only_available=0&sSortType=comment&sSortFlag=DESC"
            for p in range(2, pgcnt):
                url = prefix % (response.meta['cid'], p)
                yield scrapy.Request(url, callback=self.parseHotelForPage, meta=response.meta)

    def parseResForPage(self, response):
        cylist = response.selector.xpath('/html/body/div[2]/div[3]/div/div[2]/ul/li')
        for cy in cylist:
            res = items.MaRes()
            res['cid'] = response.meta['cid']
            res['cname'] = cy.xpath('.//div[@class="title"]/h3/a/text()').extract()
            res['url'] = cy.xpath('.//div[@class="title"]/h3/a/@href').extract()
            res['imgurl'] = cy.xpath('.//span/a/img/@src').extract()
            res['score'] = cy.xpath('.//div[@class="grade"]/em/text()').extract()
            yield res


    def parseRes(self, response):
        pid = 1
        cylist = response.selector.xpath('/html/body/div[2]/div[3]/div/div[2]/ul/li')
        for cy in cylist:
            res = items.MaRes()
            res['cid'] = response.meta['cid']
            res['cname'] = cy.xpath('.//div[@class="title"]/h3/a/text()').extract()
            res['url'] = cy.xpath('.//div[@class="title"]/h3/a/@href').extract()
            res['imgurl'] = cy.xpath('.//span/a/img/@src').extract()
            res['score'] = cy.xpath('.//div[@class="grade"]/em/text()').extract()
            yield res

        pgnum = response.selector.xpath('//div[@class="page-hotel"]/span[@class="count"]/span/text()').extract()
        if len(pgnum) > 0:
            pgcnt = int(pgnum[0])
            prefix = "http://www.mafengwo.cn/cy/%s/0-0-0-0-0-%d.html"
            for p in range(2, pgcnt):
                url = prefix % (response.meta['cid'], p)
                yield scrapy.Request(url, callback=self.parseResForPage, meta=response.meta)

    def parseMdd(self, response):
        pass

    def parse(self, response):
        #chinaMDD = response.selector.xpath('/html/body/div[2]/div[3]/div[2]/div/dl')
        chinaMDD = response.selector.xpath('//div[@class="bd bd-china"]/dl')
        youji_prefix = "http://www.mafengwo.cn/yj/%s"
        jd_prefix = "http://www.mafengwo.cn/jd/%s"
        hotel_prefix = "http://www.mafengwo.cn/hotel/%s"
        res_prefix = "http://www.mafengwo.cn/cy/%s"
        for mdds in chinaMDD:
            mddlist = mdds.xpath('dd/ul/li')
            for mddItem in mddlist:
                mdd = items.MaMDD()
                mdd['cname'] = mddItem.xpath('a/text()').extract()[0]
                url = mddItem.xpath('a/@href').extract()[0]
                mdd['url'] = url
                end = url.rfind('.html')
                start = url.rfind('/')
                cid = url[start+1:end]
                mdd['cid'] = cid 
                yield mdd
                tmpurl = self.prefix + mdd['url']
                #yield scrapy.Request(tmpurl, callback=self.parseMdd, meta={'cid': cid})
                turl  = youji_prefix % cid
                yield scrapy.Request(turl, callback=self.parseYouji, meta={'cid': cid}, dont_filter=True)
                turl = jd_prefix % cid
                #yield scrapy.Request(turl, callback=self.parseJD, meta={'cid': cid})
                turl = hotel_prefix % cid
                #yield scrapy.Request(turl, callback=self.parseHotel, meta={'cid': cid})

                turl = res_prefix % cid
                yield scrapy.Request(turl, callback=self.parseRes, meta={'cid': cid})
        #globalMDD = response.selector.xpath('/html/body/div[2]/div[3]/div[3]/div/dl')
        globalMDD = response.selector.xpath('//div[@class="row-list"][2]/div[@class="bd"]/dl')
        for mdds in globalMDD:
            mddlist = mdds.xpath('dd/ul/li/a')
            for mddItem in mddlist:
                print mddItem.extract()
                mdd = items.MaMDD()
                cname= mddItem.xpath('text()').extract()
                ename = mddItem.xpath('span/text()').extract()
                #ename = mddItem.extract()
                url = mddItem.xpath('@href').extract()
                if len(cname) > 0 and len (ename) > 0 and len(url) > 0:
                    mdd['cname'] = cname[0]
                    mdd['ename'] = ename[0]
                    mdd['url'] = url[0]
                    end = url[0].rfind('.html')
                    start = url[0].rfind('/')
                    cid = url[0][start+1:end]
                    mdd['cid'] = cid 
                    yield mdd
                    print mdd
                    tmpurl = self.prefix + mdd['url']
                    #yield scrapy.Request(tmpurl, callback=self.parseMdd, meta={'cid': cid})
                    turl  = youji_prefix % cid
                    yield scrapy.Request(youji_prefix % cid, callback=self.parseYouji, meta={'cid': cid}, dont_filter=True)
                    turl = jd_prefix % cid
                    #yield scrapy.Request(turl, callback=self.parseJD, meta={'cid': cid})
                    turl = hotel_prefix % cid
                    #yield scrapy.Request(turl, callback=self.parseHotel, meta={'cid': cid})

                    turl = res_prefix % cid
                    #yield scrapy.Request(turl, callback=self.parseRes, meta={'cid': cid})
